<?php
require_once "common.php";

_write_html_frame('采集niceui网页');
_Log("开始采集niceui网页", 'note');


define('NICEUI_HOST', 'www.niceui.cn');
define('NICEUI_HOME', 'http://' . NICEUI_HOST . '/');
define('STATIC_FIX', '.html');

$index = NICEUI_HOME;


$urls = array (
    array (
        'href' => NICEUI_HOME,
        'referer' => NICEUI_HOME,
    )
);

while ( $url = array_pop($urls) ) {
    $href = $url['href'];
    if ( !collected($href) ) {


        $referer = $url['referer'];

        _Log("开始采集网页:" . $href, 'note');
        $_content = _request($href, $referer, 86400);
        if ( $_content ) {
            _Log("成功采集网页:" . $href, 'success');
            phpQuery::newDocumentHTML($_content);
            $anchors = pq('a');
            $anchor_size = $anchors->size();
            _Log("页面中包含{$anchor_size}个连接", 'note');

            for ( $i = 0; $i < $anchor_size; $i++ ) {
                $anchor_url = $anchors->eq($i)->attr('href');
                if ( isNUrl($anchor_url) && !collected($anchor_url) ) {
                    $anchor_url = niceuiStaticUrl($anchor_url);
                    var_dump($anchor_url);
                    array_push($urls, array (
                        'href' => $anchor_url,
                        'referer' => $href
                    ));
                }
            }
            $anchors = pq('script');
            $anchor_size = $anchors->size();
            _Log("页面中包含{$anchor_size}个script连接", 'note');

            for ( $i = 0; $i < $anchor_size; $i++ ) {
                $anchor_url = $anchors->eq($i)->attr('src');
                if ( isNUrl($anchor_url) && !collected($anchor_url) ) {
                    $anchor_url = niceuiStaticUrl($anchor_url);
                    var_dump($anchor_url);
                    array_push($urls, array (
                        'href' => $anchor_url,
                        'referer' => $href
                    ));
                }
            }
            $anchors = pq('link');
            $anchor_size = $anchors->size();
            _Log("页面中包含{$anchor_size}个link连接", 'note');

            for ( $i = 0; $i < $anchor_size; $i++ ) {
                $anchor_url = $anchors->eq($i)->attr('src');
                if ( isNUrl($anchor_url) && !collected($anchor_url) ) {
                    $anchor_url = niceuiStaticUrl($anchor_url);
                    var_dump($anchor_url);
                    array_push($urls, array (
                        'href' => $anchor_url,
                        'referer' => $href
                    ));
                }
            }
            if ( savePageToLocal($href, $_content) ) {
                _Log("成功保存网页:" . $href, 'success');
            } else {
                _Log("错误保存网页:" . $href, 'error');
            }
        } else {

            _Log("采集失败网页:" . $href, 'error');
        }
    } else {
        _Log("已经采集过了:" . $href, 'error');
    }


}
/*
niceuiStaticUrl('http://www.niceui.cn/html/news/index.html');
niceuiStaticUrl('http://www.niceui.cn/?c=services&a=index');
niceuiStaticUrl('http://www.niceui.cn/?c=cases&a=view&nid=25#123');
niceuiStaticUrl('http://www.niceui.cn/?c=news&a=index');
niceuiStaticUrl('http://www.niceui.cn/?c=news&a=casenews&nid=28');
niceuiStaticUrl('http://www.niceui.cn/?c=blogs&a=view&nid=43');
*/
function collected($url)
{
    if ( is_file(getLocalPath($url)) ) {
        return true;
    }
    return false;
}

function isNUrl($url)
{
    if ( _isURL($url) ) {
        $purl = parse_url($url);
        if ( $purl['host'] == NICEUI_HOST ) {
            return true;
        }
    }
    return false;
}

function niceuiStaticUrl($url)
{
    $purl = parse_url($url);
    if ( isset($purl['query']) ) {
        parse_str($purl['query'], $query);
        //var_dump($query);
        $purl['path'] = '/html/';///html/services/index.html
        if ( isset($query['c']) ) {
            $purl['path'] .= '/' . $query['c'];
        }
        if ( isset($query['a']) ) {
            $purl['path'] .= '/' . $query['a'];
        }
        if ( isset($query['nid']) ) {
            $purl['path'] .= '-' . $query['nid'];
        }
        $purl['path'] .= STATIC_FIX;
        unset($purl['query']);
        $url = '';
        if ( isset($purl['scheme']) ) {
            $url .= $purl['scheme'] . '://';
        }
        if ( isset($purl['host']) ) {
            $url .= $purl['host'];
        }
        if ( isset($purl['path']) ) {
            $url .= '/' . $purl['path'];
        }
        if ( isset($purl['query']) ) {
            $url .= '?' . $purl['query'];
        }
        if ( isset($purl['fragment']) ) {
            $url .= '#' . $purl['fragment'];
        }
    }
    $url = formatUrl($url);
    return $url;
}

function niceuiUrlToPath($url)
{
    $purl = parse_url($url);
    if ( !isset($purl['path']) || $purl['path'] == '/' || $purl['path'] == '' ) {
        $purl['path'] = '/html/index' . STATIC_FIX;
    }
    return $purl['host'] . $purl['path'];
}

function savePageToLocal($url, $content)
{
    $filename = getLocalPath($url);
    $dirname = dirname($filename);
    if ( !is_dir($dirname) )
        mkdir($dirname, 0777, true);
    return file_put_contents($filename, $content);
}

function getLocalPath($url)
{
    $path = niceuiUrlToPath($url);
    return BASE_PATH . 'data/' . $path;
}